{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32b2d9d1-6bda-40d5-871e-12a7c616d8e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"THUDM/ChatGLM3-6B\"\n",
    "train_data_path = 'HasturOfficial/adgen'\n",
    "eval_data_path = None\n",
    "seed = 8\n",
    "max_input_len = 512\n",
    "max_output_len = 1536\n",
    "lora_rank = 4\n",
    "lora_alpha = 32\n",
    "lora_dropout = 0.05\n",
    "resume_from_checkpoint = None\n",
    "prompt_text = ''\n",
    "compute_dtype = 'fp32'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a340a66c-7ab3-4451-807c-851db2e88212",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset(train_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7bdea935-bebb-4446-9058-0c702ba90ebc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['content', 'summary'],\n",
       "        num_rows: 114599\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['content', 'summary'],\n",
       "        num_rows: 1070\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2b9e7dbf-84ee-4bda-983f-6a8f14c3225b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import ClassLabel, Sequence\n",
    "import random\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "def show_random_elements(dataset, num_examples=10):\n",
    "    assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n",
    "    picks = []\n",
    "    for _ in range(num_examples):\n",
    "       pick = random.randint(0, len(dataset)-1)\n",
    "       while pick in picks:\n",
    "           pick = random.randint(0, len(dataset)-1)\n",
    "       picks.append(pick)\n",
    "    df = pd.DataFrame(dataset[picks])\n",
    "    for column, typ in dataset.features.items():\n",
    "        if isinstance(typ, ClassLabel):\n",
    "            df[column] = df[column].transform(lambda i: typ.names[i])\n",
    "        elif isinstance(typ, Sequence) and isinstance(typ.feature, ClassLabel):\n",
    "            df[column] = df[column].transform(lambda x: [typ.feature.names[i] for i in x])\n",
    "    display(HTML(df.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aaa4127b-458e-48b3-8245-4a3e4c540068",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>类型#裙*版型#宽松*版型#显瘦*颜色#黑色*图案#条纹*裙型#鱼尾裙*裙下摆#荷叶边*裙款式#螺纹</td>\n",
       "      <td>本品版型宽松，造型独特时尚。竖坑条设计，拉伸黑色条纹，视觉显瘦感好，凸显你的优雅好身材。浪漫荷叶边设计，裙摆蹁跹优雅，螺纹搭配鱼尾，美观又蓬松，给身体预留出足够的空间，穿着的同时，可以尽情展现自己。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>类型#上衣*风格#简约*图案#抽象*图案#印花*衣样式#衬衫*衣领型#立领*衣领型#翻领*衣袖长#长袖*衣门襟#单排扣</td>\n",
       "      <td>这款来自BRAND旗下精心推出的男士长袖衬衫，前幅利用简约的抽象印花图案修饰，增添整体的时尚气质，又具有别样的迷人气质。经典的立领翻领领口，立体感十足，也让衣物廓形更明晰。时髦的单排扣衣襟，穿脱很便利，展露出温文尔雅的气息，做工与剪裁属于一流。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>类型#裤*版型#显瘦*材质#牛仔布*风格#复古*风格#性感*图案#复古*图案#线条*裤款式#口袋*裤腰型#高腰*裤口#开叉*裤口#微喇裤</td>\n",
       "      <td>这是一款修身版型的牛仔裤，流畅的裁剪线条，打造出的开衩设计，带有轻柔飘逸的质感，让你在帅气与优雅中随意的切换，还不失性感魅力。经典的喇叭裤造型，演绎复古时尚。高腰的设计，凸显腰线的位置，视觉上秒变大长腿。对称性的斜插口袋，方便放置物品。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>类型#裙*颜色#黑色*风格#高贵*风格#清新*裙型#蛋糕*裙下摆#层叠</td>\n",
       "      <td>繁复而美好的层叠设计让这款蛋糕裙有着清新而温婉的少女气息。浅粉的配色是永不过时的少女梦想，甜蜜而唯美。黑色的配色则有着高贵低调的韵味，是贵族小姐的冷艳美感。飘逸的下摆柔美缱绻，温柔动人。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>类型#裙*版型#显瘦*材质#牛仔布*风格#复古*风格#性感*图案#复古*裙型#牛仔裙*裙型#包臀裙*裙下摆#开叉*裙下摆#毛边*裙长#半身裙</td>\n",
       "      <td>牛仔半裙的出&lt;UNK&gt;相信是每个mm都知道的，这是一款既复古又有范的牛仔半裙，全手工的漆点尽显个性和潇洒，为半裙增添满满的不羁气质。半裙是修身的包臀版型设计，视觉上尽显你的曼妙身材，也性感又抢眼。裙身有一道开叉设计，搭配下摆的毛边设计，更添率性气息。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>类型#裙*材质#蕾丝*风格#知性*风格#性感*图案#蕾丝*裙型#小黑裙*裙领型#圆领*裙款式#拼接*裙款式#勾花镂空</td>\n",
       "      <td>好像小黑裙总会给人一种很神秘很妩媚的感觉，裙身采用秀气的圆领设计，贴合颈部，凸显知性优雅，展现女性的天鹅颈。以及领口设计了小镂空的裁剪，微露肌肤，平添了不少性感韵味。肩部采用蕾丝的拼接，显得甜美带洋气的气息。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>类型#上衣*版型#宽松*材质#网纱*颜色#白色*风格#复古*风格#性感*图案#斑马纹*图案#复古*图案#创意*衣样式#卫衣*衣款式#拼接</td>\n",
       "      <td>这款由黑白色打造的卫衣，打破传统卫衣的局限性，衣袖处采用网纱拼接，复古优雅中充满着性感魅惑。宽窄不一的斑马纹，打破世俗的旧看法，创意无限让人浮想联翩。上身的效果宽松舒适，却能勾勒出前凸后翘的好身材，尤显做工精致。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>类型#裙*材质#网纱*裙型#蛋糕*裙型#抹胸裙*裙长#连衣裙*裙款式#亮片</td>\n",
       "      <td>这款连衣裙第一眼就美得让人窒息，在温柔的网纱织面上，点缀了炫目晶莹的亮片元素，看起来层次丰富而梦幻，流露出的朦胧感特别美妙，颇具华丽隆重的贵族气息。甜美的抹胸式设计更加有女人味，可以尽情展现女生的曼妙身姿。三层蛋糕裙摆仙气满满，&lt;UNK&gt;着每一位有着少女心的girl，简单一件就能让你秒变小公举。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>类型#上衣*版型#宽松*版型#显瘦*风格#清新*图案#条纹*衣样式#衬衫</td>\n",
       "      <td>这款衬衣，的清新蓝白条纹十分给人自然干净的感觉，大方时尚条纹还有显瘦的功能。胸口的设计很特别，胸口的门襟铜扣，十分有质感，一看就很有品质，有着点睛的作用。宽松的版型，遮肉功能十分强大更显强调。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>类型#上衣*版型#显瘦*材质#羊毛*图案#印花*衣样式#毛衣*衣领型#圆领*衣袖长#长袖*衣门襟#套头</td>\n",
       "      <td>来自BRAND，黑豹嵌花毛衣。精选100％羊毛材质打造，软糯轻薄，穿着透气。简洁小圆领长袖套头款式设计，略微修身的版型作为内搭或是外穿皆出彩。个性黑豹印花图案装饰，彰显霸气设计细节。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>类型#上衣*风格#简约*风格#知性*图案#条纹*图案#线条*衣样式#衬衫*衣领型#一字领*衣款式#勾花镂空</td>\n",
       "      <td>sitiselected这款条纹一字领衬衫，简约的一字领设计，尽显优雅知性。镂空排扣袖设计，修饰手臂的额线条。下摆两侧开衩裁剪，方便穿着，提升整衣的细节感。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>类型#裤*版型#宽松*材质#牛仔布*材质#水洗*风格#复古*风格#休闲*图案#复古*裤长#长裤*裤型#直筒裤*裤款式#拼接*裤款式#破洞*裤款式#不规则*裤腰型#高腰</td>\n",
       "      <td>BRAND带来的这款长裤，后幅采用解构式双腰头设计，加以小心机的高腰处理，能够有效提高腰线；前后拼接结合水洗磨白工艺，带来富有层次感的复古牛仔视效；宽松的直筒裤型，对身材有良好的包容性，打造休闲随性的穿着视效；加以裤身不规则的破洞设计，尽显叛逆不羁的个性。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>类型#上衣*图案#字母*图案#格子*图案#文字*图案#线条*图案#印花*图案#撞色*衣样式#衬衫*衣领型#立领*衣款式#纽扣</td>\n",
       "      <td>这件印花立领衬衫，在洁白的底色上以撞色字母印花装饰，各字母之间以线条&lt;UNK&gt;格子图案，使衣衫极具时尚大方的感觉，直筒的版型设计，修饰身材的同时，也能有一定的包容性，能有效的遮掩小肚&lt;UNK&gt;上的赘肉，后背领口处开叉设计。以纽扣开合，方便穿脱。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>类型#裤*材质#牛仔布*裤长#短裤*裤腰型#高腰*裤口#毛边</td>\n",
       "      <td>牛仔裤总是女孩子们非常关注的裤子款式之一。高腰的牛仔短裤，可以拉长女性的身材曲线，让自己的双腿变得更完美哦。再利用毛边进行点缀，也可以彰显出自己的随性美感呀，不用担心自己穿着很普通啦。又利用防走光的设计，更可以给自己带来保守气质哦。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>类型#裙*颜色#纯色*风格#淑女*风格#简约*图案#纯色*图案#刺绣</td>\n",
       "      <td>名副其实的淑女裙，缤纷的彩绣图案赋予其几丝民族风情，精致且令人惊艳。绣花灵活而生动的装饰上半身，如百花盛放般绚烂，叫人欣赏不够。裙摆则以纯色来演绎，进行简约的碰撞，视觉感浪漫唯美。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>类型#裙*版型#显瘦*图案#线条*裙型#背带裙*裙领型#v领</td>\n",
       "      <td>v字的领型是在背带裙的设计当中经常会出现的领型，其中一个重要原因就是v字领型的修身效果跟背带裙个性相搭，用在背带裙的设计上十分应景。而且v字的领型线条比较利落，它有着视觉上面的拉长感，让女孩子轻松穿出显高显瘦的效果。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>类型#上衣*颜色#红色*风格#青春*衣样式#外套*衣长#短款*衣款式#口袋</td>\n",
       "      <td>这款外套对于个子矮小的妹纸来说就是福音了，短款穿在身上搭配起起来，立马就能变成大长腿，把整体身长比例拉长，呈现出黄金比例效果。鲜艳活泼的红色，穿在身上，视觉上给人呈现出青春的活力，元气满满的少女，还能衬托出肌肤的白皙，拥有一整天的好气色。大大的口袋，既可以作为装饰，出门携带东西也是非常的方便，还能增加整体的层次感。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>类型#裙*材质#网纱*颜色#白色*裙款式#吊带</td>\n",
       "      <td>裙子采用两件套设计，内里为长款吊带，外罩选用珍珠点缀的网纱，精致的做工与纯洁的白色相得益彰，素雅纯洁仙气十足~网纱若隐若现的视觉感受与吊带元素提升了整个人成熟的气质。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>类型#裤*风格#复古*图案#蝴蝶结*图案#复古*图案#线条*裤型#阔腿裤*裤款式#松紧带</td>\n",
       "      <td>设计师新颖的将金丝绒面料与阔腿裤的版型相结合，赋予了裤装几分复古的风情，垂顺的面料呈现出拉长腿部线条的视觉效果，更显高挑纤细身姿。松紧带设计，即穿脱更方便，又令其舒适没有束缚感。裤头蝴蝶结系带点缀，尽显女性闲适的自在感。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>类型#裙*风格#青春*裙腰型#高腰*裙长#短裙*裙款式#绑带</td>\n",
       "      <td>这条高腰短裙，看似是短裙，其实内里还做了短裤的设计，时髦感强却又不会显得太浮夸。高腰的廓形设计，能够使得腿部看起来更加的修长。裙身做了钉扣的绑带设计，散发着青春甜美的气息。</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_random_elements(dataset['validation'], num_examples=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "76abe41c-b6d9-4144-ad9e-eb46e0d23e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_func(example, tokenizer, ignore_label_id=-100):\n",
    "    question = prompt_text + example['content']\n",
    "    if example.get('input', None) and example['input'].strip():\n",
    "        question += f'\\n{example[\"input\"]}'\n",
    "        \n",
    "    answer = example['summary']\n",
    "    q_ids = tokenizer.encode(text=question, add_special_tokens=False)\n",
    "    a_ids = tokenizer.encode(text=answer, add_special_tokens=False)\n",
    "\n",
    "    if len(q_ids) > max_input_len - 2:  \n",
    "        q_ids = q_ids[:max_input_len - 2]\n",
    "    if len(a_ids) > max_output_len - 1:  \n",
    "        a_ids = a_ids[:max_output_len - 1]\n",
    "\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(q_ids, a_ids)\n",
    "    question_length = len(q_ids) + 2  \n",
    "\n",
    "    labels = [ignore_label_id] * question_length + input_ids[question_length:]\n",
    "\n",
    "    return {'input_ids': input_ids, 'labels': labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a289bf2-c447-419b-a11b-ad79924139ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,revision='b098244' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "97cd3f7b-f42f-40b4-aec8-981687fc1e22",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = dataset['train'].column_names\n",
    "tokenized_dataset = dataset['train'].map(\n",
    "    lambda example: tokenize_func(example, tokenizer),\n",
    "    batched=False,\n",
    "    remove_columns=column_names\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cb528242-7eac-49af-9fec-4750de2ff9fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_ids</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56532, 30998, 55090, 54888, 31010, 40833, 30998, 38317, 31010, 57007, 30998, 33692, 31010, 55906, 54785, 30998, 32799, 31010, 40512, 30998, 37505, 31010, 55906, 54785, 30998, 56532, 54888, 31010, 54839, 57449, 56532, 30998, 56532, 54888, 31010, 55379, 54882, 56532, 30910, 31773, 55115, 55379, 54882, 56532, 55906, 54785, 54827, 54862, 31123, 33454, 54733, 55014, 31799, 52392, 31155, 31828, 40512, 54530, 55090, 54735, 33762, 54657, 33454, 31123, 43729, 54733, 55014, 34119, 54706, 31123, 33762, 33481, 38395, 31155, 40833, 54839, 57449, 31123, 54535, 55679, 34319, 36107, 42144, 31155, 54813, 55679, 55114, 54878, 54530, 57007, 54824, 54901, 54980, 31123, 55113, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 31773, 55115, 55379, 54882, 56532, 55906, 54785, 54827, 54862, 31123, 33454, 54733, 55014, 31799, 52392, 31155, 31828, 40512, 54530, 55090, 54735, 33762, 54657, 33454, 31123, 43729, 54733, 55014, 34119, 54706, 31123, 33762, 33481, 38395, 31155, 40833, 54839, 57449, 31123, 54535, 55679, 34319, 36107, 42144, 31155, 54813, 55679, 55114, 54878, 54530, 57007, 54824, 54901, 54980, 31123, 55113, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56778, 30998, 55090, 54888, 31010, 49899, 30998, 33692, 31010, 34198, 30998, 32799, 31010, 40589, 30998, 56778, 54888, 31010, 30913, 54952, 30998, 56778, 54888, 31010, 57069, 57069, 56778, 30998, 56778, 54578, 56164, 31010, 56262, 55540, 55086, 30998, 56778, 54578, 56164, 31010, 55180, 56940, 30998, 56778, 56278, 54888, 31010, 54589, 56278, 30998, 56778, 54625, 31010, 55205, 54715, 56778, 30998, 56778, 40877, 31010, 56897, 54882, 30910, 44644, 54530, 56897, 54882, 45588, 54557, 54642, 36259, 56420, 55569, 54542, 54986, 56432, 31123, 40479, 58521, 54533, 33537, 31123, 39816, 54530, 56420, 55031, 33554, 43007, 31155, 55180, 56940, 54530, 56262, 55540, 55086, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 44644, 54530, 56897, 54882, 45588, 54557, 54642, 36259, 56420, 55569, 54542, 54986, 56432, 31123, 40479, 58521, 54533, 33537, 31123, 39816, 54530, 56420, 55031, 33554, 43007, 31155, 55180, 56940, 54530, 56262, 55540, 55086, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56778, 30998, 55090, 54888, 31010, 49899, 30998, 33692, 31010, 34298, 30998, 37505, 31010, 39424, 54784, 30998, 56778, 54888, 31010, 35119, 56778, 30998, 56778, 54578, 56164, 31010, 56262, 55540, 55086, 30998, 56778, 54625, 31010, 55205, 54715, 56778, 30998, 56778, 55500, 54811, 58709, 31010, 54712, 54882, 30910, 33730, 55205, 54715, 56778, 49950, 34298, 46839, 34523, 32339, 36962, 31123, 46839, 33894, 41169, 40398, 31123, 56597, 55857, 33638, 54607, 54618, 31123, 35752, 54530, 34298, 54687, 32307, 56778, 54888, 33804, 34219, 35515, 31123, 44519, 55108, 35879, 52537, 31155, 56278, 54655, 54532, 39424, 54784, 54712, 54882, 55101, 54557, 56278, 54715, 31123, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 33730, 55205, 54715, 56778, 49950, 34298, 46839, 34523, 32339, 36962, 31123, 46839, 33894, 41169, 40398, 31123, 56597, 55857, 33638, 54607, 54618, 31123, 35752, 54530, 34298, 54687, 32307, 56778, 54888, 33804, 34219, 35515, 31123, 44519, 55108, 35879, 52537, 31155, 56278, 54655, 54532, 39424, 54784, 54712, 54882, 55101, 54557, 56278, 54715, 31123, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56532, 30998, 32799, 31010, 40512, 30998, 56532, 40877, 31010, 55919, 57278, 30998, 56532, 56278, 54888, 31010, 55426, 55316, 56278, 30910, 36139, 33165, 55033, 55091, 31893, 55821, 55428, 55428, 31123, 56532, 54715, 32195, 54620, 54715, 56597, 55857, 31123, 40231, 37003, 33550, 54530, 35405, 32144, 31155, 33923, 54633, 49244, 32802, 54568, 34511, 55426, 55316, 55919, 57278, 54882, 31123, 34068, 54730, 31992, 35066, 44654, 54554, 33267, 56278, 55197, 56291, 56068, 32184, 54659, 31701, 54548, 40512, 54827, 54862, 54825, 54533, 31123, 33313, 41076, 54999, 55583, 43949, 31155, 2]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 36139, 33165, 55033, 55091, 31893, 55821, 55428, 55428, 31123, 56532, 54715, 32195, 54620, 54715, 56597, 55857, 31123, 40231, 37003, 33550, 54530, 35405, 32144, 31155, 33923, 54633, 49244, 32802, 54568, 34511, 55426, 55316, 55919, 57278, 54882, 31123, 34068, 54730, 31992, 35066, 44654, 54554, 33267, 56278, 55197, 56291, 56068, 32184, 54659, 31701, 54548, 40512, 54827, 54862, 54825, 54533, 31123, 33313, 41076, 54999, 55583, 43949, 31155, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 49534, 30998, 55090, 54888, 31010, 49899, 30998, 32799, 31010, 44785, 30998, 32799, 31010, 31900, 54803, 30998, 37505, 31010, 44785, 30998, 37505, 31010, 55845, 57435, 30998, 55500, 46025, 31010, 42373, 30998, 55500, 56896, 54625, 31010, 54625, 56896, 30910, 57435, 54867, 40326, 54625, 56896, 49534, 31123, 36349, 34902, 54662, 51605, 32985, 33069, 54704, 54586, 54878, 31155, 44610, 54625, 56896, 54838, 55432, 33804, 41945, 56064, 56278, 31123, 44785, 55025, 35752, 31123, 32985, 31901, 42373, 31677, 33638, 50927, 54557, 46633, 54539, 33941, 34573, 54619, 34219, 31155, 56319, 33711, 54867, 55540, 55845, 57435, 35765, 54557, 44785, 31900, 54803, 31123, 51480, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 57435, 54867, 40326, 54625, 56896, 49534, 31123, 36349, 34902, 54662, 51605, 32985, 33069, 54704, 54586, 54878, 31155, 44610, 54625, 56896, 54838, 55432, 33804, 41945, 56064, 56278, 31123, 44785, 55025, 35752, 31123, 32985, 31901, 42373, 31677, 33638, 50927, 54557, 46633, 54539, 33941, 34573, 54619, 34219, 31155, 56319, 33711, 54867, 55540, 55845, 57435, 35765, 54557, 44785, 31900, 54803, 31123, 51480, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56778, 30998, 33692, 31010, 55168, 32632, 30998, 56778, 54625, 31010, 54625, 56778, 30998, 56778, 55500, 54811, 58709, 31010, 54712, 54882, 30998, 56778, 40877, 31010, 55919, 57278, 30998, 56778, 40877, 31010, 54805, 55281, 30910, 55168, 32632, 56363, 55140, 54542, 56225, 54868, 54947, 54610, 31123, 32985, 40402, 47274, 56363, 54882, 31123, 55168, 32632, 55919, 57278, 54712, 54882, 37135, 33870, 31155, 56363, 54610, 42128, 54805, 55281, 33162, 31123, 40479, 41484, 31900, 44785, 54803, 31123, 35759, 32985, 54625, 56778, 31716, 55354, 56532, 54606, 54835, 31741, 56331, 42845, 31155, 2]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 55168, 32632, 56363, 55140, 54542, 56225, 54868, 54947, 54610, 31123, 32985, 40402, 47274, 56363, 54882, 31123, 55168, 32632, 55919, 57278, 54712, 54882, 37135, 33870, 31155, 56363, 54610, 42128, 54805, 55281, 33162, 31123, 40479, 41484, 31900, 44785, 54803, 31123, 35759, 32985, 54625, 56778, 31716, 55354, 56532, 54606, 54835, 31741, 56331, 42845, 31155, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 49534, 30998, 38317, 31010, 58186, 55599, 30998, 38317, 31010, 55534, 57375, 30998, 32799, 31010, 44785, 30998, 32799, 31010, 40589, 30998, 37505, 31010, 39424, 54784, 30998, 37505, 31010, 44785, 30998, 55500, 46025, 31010, 55534, 57375, 57439, 30998, 55500, 40877, 31010, 55919, 59023, 30910, 52124, 54530, 55534, 57375, 57439, 32985, 54547, 54847, 40419, 38084, 44785, 59023, 57688, 31735, 31123, 44517, 45944, 31716, 55090, 54888, 31155, 41117, 33551, 36925, 54666, 54882, 32056, 55124, 44785, 35752, 44750, 54735, 32799, 31123, 35878, 40894, 32745, 54727, 55072, 55487, 32048, 40589, 51622, 31123, 39424, 54784, 54530, 56896, 54815, 31735, 31123, 54687, 35752, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 52124, 54530, 55534, 57375, 57439, 32985, 54547, 54847, 40419, 38084, 44785, 59023, 57688, 31735, 31123, 44517, 45944, 31716, 55090, 54888, 31155, 41117, 33551, 36925, 54666, 54882, 32056, 55124, 44785, 35752, 44750, 54735, 32799, 31123, 35878, 40894, 32745, 54727, 55072, 55487, 32048, 40589, 51622, 31123, 39424, 54784, 54530, 56896, 54815, 31735, 31123, 54687, 35752, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 49534, 30998, 38317, 31010, 38683, 54901, 30998, 33692, 31010, 54829, 35798, 30998, 37505, 31010, 33242, 30998, 55500, 46025, 31010, 51746, 30998, 55500, 46025, 31010, 42373, 30910, 32307, 55090, 54888, 32698, 38395, 31123, 54603, 54839, 57449, 56532, 54888, 31123, 51605, 33454, 54733, 55014, 54535, 56593, 59402, 31123, 54585, 56278, 55350, 55184, 34394, 31123, 32115, 33621, 31735, 31123, 33481, 34372, 31123, 35400, 32286, 34002, 54530, 35489, 31123, 33168, 35237, 38638, 33481, 54706, 31155, 33080, 31687, 31123, 54829, 35798, 55602, 31002, 5234, 30984, 30994, 54785, 31123, 55602, 54683, 33454, 31123, 46839, 55698, 54596, 31123, 51605, 33894, 39819, 31123, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 32307, 55090, 54888, 32698, 38395, 31123, 54603, 54839, 57449, 56532, 54888, 31123, 51605, 33454, 54733, 55014, 54535, 56593, 59402, 31123, 54585, 56278, 55350, 55184, 34394, 31123, 32115, 33621, 31735, 31123, 33481, 34372, 31123, 35400, 32286, 34002, 54530, 35489, 31123, 33168, 35237, 38638, 33481, 54706, 31155, 33080, 31687, 31123, 54829, 35798, 55602, 31002, 5234, 30984, 30994, 54785, 31123, 55602, 54683, 33454, 31123, 46839, 55698, 54596, 31123, 51605, 33894, 39819, 31123, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56532, 30998, 37505, 31010, 56483, 54785, 30998, 56532, 54625, 31010, 55354, 56532, 30998, 56532, 54888, 31010, 54839, 57449, 56532, 30998, 56532, 40877, 31010, 56416, 56241, 30998, 56532, 54815, 31010, 55693, 55086, 30910, 38984, 55906, 55790, 54530, 33106, 46839, 31123, 35081, 56487, 54588, 54706, 32760, 33730, 55354, 56532, 35212, 34481, 54706, 31123, 54534, 35405, 32300, 42462, 31923, 39819, 55113, 56089, 31123, 54839, 57449, 54530, 55354, 56532, 33561, 31123, 35381, 35405, 56165, 56558, 53161, 33253, 51827, 34006, 55069, 55411, 34319, 31155, 56532, 56158, 54807, 55693, 55086, 34481, 31123, 35878, 54706, 35212, 31155, 58276, 33800, 56483, 54785, 56416, ...]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 38984, 55906, 55790, 54530, 33106, 46839, 31123, 35081, 56487, 54588, 54706, 32760, 33730, 55354, 56532, 35212, 34481, 54706, 31123, 54534, 35405, 32300, 42462, 31923, 39819, 55113, 56089, 31123, 54839, 57449, 54530, 55354, 56532, 33561, 31123, 35381, 35405, 56165, 56558, 53161, 33253, 51827, 34006, 55069, 55411, 34319, 31155, 56532, 56158, 54807, 55693, 55086, 34481, 31123, 35878, 54706, 35212, 31155, 58276, 33800, 56483, 54785, 56416, ...]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[64790, 64792, 30910, 33467, 31010, 56532, 30998, 32799, 31010, 34435, 30998, 56532, 54888, 31010, 56529, 56158, 56532, 30998, 56532, 56278, 54888, 31010, 54589, 56278, 30998, 56532, 54815, 31010, 55072, 57802, 56532, 30910, 33730, 56529, 56158, 56532, 38984, 33106, 39145, 54980, 31123, 31892, 54718, 54706, 33894, 31123, 55432, 54534, 32794, 54656, 31685, 55113, 56089, 53161, 31155, 54589, 56278, 54530, 40877, 31803, 56278, 54831, 33253, 33780, 55097, 54625, 34319, 32938, 31155, 56529, 56158, 34481, 31677, 35002, 36241, 31123, 34435, 54892, 43386, 31155, 44926, 56532, 34481, 33780, 31942, 43518, 56158, 54888, 31155, 2]</td>\n",
       "      <td>[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 30910, 33730, 56529, 56158, 56532, 38984, 33106, 39145, 54980, 31123, 31892, 54718, 54706, 33894, 31123, 55432, 54534, 32794, 54656, 31685, 55113, 56089, 53161, 31155, 54589, 56278, 54530, 40877, 31803, 56278, 54831, 33253, 33780, 55097, 54625, 34319, 32938, 31155, 56529, 56158, 34481, 31677, 35002, 36241, 31123, 34435, 54892, 43386, 31155, 44926, 56532, 34481, 33780, 31942, 43518, 56158, 54888, 31155, 2]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_random_elements(tokenized_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8da6bffc-0a57-4fa1-9960-fa79fb534cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_dataset = tokenized_dataset.shuffle(seed=seed)\n",
    "tokenized_dataset = tokenized_dataset.flatten_indices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "63c4f356-b573-4dfe-a5ca-07691472a766",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from typing import List, Dict, Optional\n",
    "\n",
    "class DataCollatorForChatGLM:\n",
    "    def __init__(self, pad_token_id:int, max_len:int=2048, ignore_label_id:int=-100):\n",
    "        self.pad_token_id = pad_token_id\n",
    "        self.ignore_label_id=ignore_label_id\n",
    "        self.max_len=max_len\n",
    "    def __call__(self, batch_data:List[Dict[str, List]]) -> Dict[str, torch.Tensor]:\n",
    "        len_list = [len(d['input_ids']) for d in batch_data]\n",
    "        batch_max_len = max(len_list)\n",
    "        input_ids, labels = [] ,[]\n",
    "        for len_of_d, d in sorted(zip(len_list, batch_data), key=lambda x: -x[0]):\n",
    "            pad_len = batch_max_len - len_of_d\n",
    "            ids = d['input_ids'] + [self.pad_token_id]* pad_len\n",
    "            label = d['labels'] + [self.ignore_label_id]* pad_len\n",
    "            if batch_max_len > self.max_len:\n",
    "                ids = ids[:self.max_len]\n",
    "                label = label[:self.max_len]\n",
    "            input_ids.append(torch.LongTensor(ids))\n",
    "            labels.append(torch.LongTensor(label))\n",
    "        input_ids = torch.stack(input_ids)\n",
    "        labels = torch.stack(labels)\n",
    "        return {'input_ids': input_ids, 'labels': labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f1b946b5-4f70-4d6f-82ed-b4d9cc4e82c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorForChatGLM(pad_token_id=tokenizer.pad_token_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "edc2c700-76bb-4b98-9904-cc54b443a3b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModel, BitsAndBytesConfig\n",
    "_compute_dtype_map = {\n",
    "    'fp32': torch.float32,\n",
    "    'fp16': torch.float16,\n",
    "    'bf16': torch.bfloat16\n",
    "}\n",
    "q_config = BitsAndBytesConfig(load_in4bit=True, bnb_4bit_quant_type='nf4',bnb_4bit_use_double_quant=True,bnb_4bit_compute_dtype=_compute_dtype_map['bf16'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c26a3a6a-4fb3-44d1-bf7a-3be7c7ca1bee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████| 7/7 [00:04<00:00,  1.41it/s]\n"
     ]
    }
   ],
   "source": [
    "model = AutoModel.from_pretrained(model_name,\n",
    "                                  quantization_config=q_config,\n",
    "                                  device_map=\"auto\",\n",
    "                                  trust_remote_code=True,revision='b098244')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "da067381-3449-4408-ae01-2adfbd204c7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.embedding.word_embeddings.weight: cuda:0\n",
      "transformer.encoder.layers.0.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.0.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.0.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.0.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.0.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.0.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.0.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.1.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.1.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.1.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.1.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.1.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.1.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.1.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.2.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.2.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.2.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.2.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.2.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.2.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.2.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.3.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.3.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.3.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.3.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.3.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.3.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.3.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.4.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.4.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.4.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.4.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.4.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.4.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.4.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.5.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.5.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.5.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.5.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.5.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.5.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.5.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.6.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.6.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.6.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.6.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.6.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.6.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.6.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.7.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.7.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.7.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.7.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.7.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.7.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.7.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.8.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.8.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.8.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.8.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.8.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.8.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.8.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.9.input_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.9.self_attention.query_key_value.weight: cuda:0\n",
      "transformer.encoder.layers.9.self_attention.query_key_value.bias: cuda:0\n",
      "transformer.encoder.layers.9.self_attention.dense.weight: cuda:0\n",
      "transformer.encoder.layers.9.post_attention_layernorm.weight: cuda:0\n",
      "transformer.encoder.layers.9.mlp.dense_h_to_4h.weight: cuda:0\n",
      "transformer.encoder.layers.9.mlp.dense_4h_to_h.weight: cuda:0\n",
      "transformer.encoder.layers.10.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.10.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.10.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.10.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.10.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.10.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.10.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.11.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.11.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.11.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.11.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.11.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.11.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.11.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.12.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.12.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.12.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.12.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.12.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.12.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.12.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.13.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.13.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.13.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.13.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.13.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.13.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.13.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.14.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.14.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.14.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.14.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.14.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.14.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.14.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.15.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.15.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.15.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.15.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.15.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.15.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.15.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.16.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.16.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.16.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.16.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.16.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.16.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.16.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.17.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.17.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.17.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.17.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.17.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.17.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.17.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.18.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.18.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.18.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.18.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.18.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.18.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.18.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.19.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.19.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.19.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.19.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.19.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.19.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.19.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.20.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.20.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.20.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.20.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.20.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.20.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.20.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.21.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.21.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.21.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.21.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.21.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.21.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.21.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.22.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.22.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.22.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.22.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.22.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.22.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.22.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.23.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.23.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.23.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.23.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.23.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.23.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.23.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.24.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.24.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.24.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.24.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.24.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.24.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.24.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.25.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.25.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.25.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.25.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.25.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.25.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.25.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.26.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.26.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.26.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.26.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.26.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.26.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.26.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.layers.27.input_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.27.self_attention.query_key_value.weight: cuda:1\n",
      "transformer.encoder.layers.27.self_attention.query_key_value.bias: cuda:1\n",
      "transformer.encoder.layers.27.self_attention.dense.weight: cuda:1\n",
      "transformer.encoder.layers.27.post_attention_layernorm.weight: cuda:1\n",
      "transformer.encoder.layers.27.mlp.dense_h_to_4h.weight: cuda:1\n",
      "transformer.encoder.layers.27.mlp.dense_4h_to_h.weight: cuda:1\n",
      "transformer.encoder.final_layernorm.weight: cuda:1\n",
      "transformer.output_layer.weight: cuda:1\n"
     ]
    }
   ],
   "source": [
    "for name , p in model.named_parameters():\n",
    "    print(f\"{name}: {p.device}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dd57db8b-d827-448a-841d-bfad632c57da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3739.69MIB\n"
     ]
    }
   ],
   "source": [
    "print(f\"{(model.get_memory_footprint()/(1024**2)):.2f}MIB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "97859d94-9419-4d81-a232-749eaacff34b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model.\n"
     ]
    }
   ],
   "source": [
    "from peft import TaskType, LoraConfig, get_peft_model, prepare_model_for_kbit_training\n",
    "kbit_model = prepare_model_for_kbit_training(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d9f3b500-fce7-4ca1-a58c-5e5027476a4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft.utils import TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING\n",
    "target_modules = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING['chatglm']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "be168cc5-31d8-4199-82d6-0717b6d07008",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['query_key_value']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fdd87e7f-4421-40be-be23-85178e7d0a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "lora_config = LoraConfig(\n",
    "    target_modules=target_modules,\n",
    "    r=lora_rank,\n",
    "    lora_alpha=lora_alpha,\n",
    "    lora_dropout=lora_dropout,\n",
    "    bias='none',\n",
    "    inference_mode=False,\n",
    "    task_type=TaskType.CAUSAL_LM\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bb9ecfca-b441-42ec-a3bf-bae60df162be",
   "metadata": {},
   "outputs": [],
   "source": [
    "qlora_model = get_peft_model(kbit_model, lora_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7f2a0a84-e69b-4a1b-81cc-b8ad67994bbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 974,848 || all params: 6,244,558,848 || trainable%: 0.0156\n"
     ]
    }
   ],
   "source": [
    "qlora_model.print_trainable_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6df78c65-6dc4-47e6-9dab-bd58ac849c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=f\"../models/{model_name}\",          # 输出目录\n",
    "    per_device_train_batch_size=16,                     # 每个设备的训练批量大小\n",
    "    gradient_accumulation_steps=4,                     # 梯度累积步数\n",
    "    per_device_eval_batch_size=8,                      # 每个设备的评估批量大小\n",
    "    max_steps = 200,\n",
    "    learning_rate=1e-3,                                # 学习率\n",
    "    num_train_epochs=1,                                # 训练轮数\n",
    "    lr_scheduler_type=\"linear\",                        # 学习率调度器类型\n",
    "    warmup_ratio=0.1,                                  # 预热比例\n",
    "    logging_steps=100,                                 # 日志记录步数\n",
    "    save_strategy=\"steps\",                             # 模型保存策略\n",
    "    save_steps=10,                                    # 模型保存步数\n",
    "    # evaluation_strategy=\"steps\",                       # 评估策略\n",
    "    # eval_steps=500,                                    # 评估步数\n",
    "    optim=\"adamw_torch\",                               # 优化器类型\n",
    "    fp16=True,                                        # 是否使用混合精度训练\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c62cda88-9ffe-4a82-bbca-ea03b08260be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=qlora_model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_dataset,\n",
    "    data_collator=data_collator\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "70b862a3-92cb-447a-b238-01ee3ecab964",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='200' max='200' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [200/200 2:13:19, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>3.640000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>3.262800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/peft/utils/other.py:716: UserWarning: Unable to fetch remote file due to the following error (ReadTimeoutError(\"HTTPSConnectionPool(host='hf-mirror.com', port=443): Read timed out. (read timeout=10)\"), '(Request ID: d2b6d242-275b-4406-b670-8577d4313131)') - silently ignoring the lookup for the file config.json in THUDM/ChatGLM3-6B.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/peft/utils/save_and_load.py:246: UserWarning: Could not find a config file in THUDM/ChatGLM3-6B - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=200, training_loss=3.4514285278320314, metrics={'train_runtime': 8029.4114, 'train_samples_per_second': 1.594, 'train_steps_per_second': 0.025, 'total_flos': 7.739107796523418e+16, 'train_loss': 3.4514285278320314, 'epoch': 0.11168504816417701})"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6c6fa6be-e5d5-4192-902d-97c54bf28564",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.model.save_pretrained(f\"../models/{model_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4e05178-6bbb-472d-874f-50aa6d645848",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=f\"../models/{model_name}\",          # 输出目录\n",
    "    per_device_train_batch_size=16,                     # 每个设备的训练批量大小\n",
    "    gradient_accumulation_steps=4,                     # 梯度累积步数\n",
    "    per_device_eval_batch_size=8,                      # 每个设备的评估批量大小\n",
    "    #max_steps = 200,\n",
    "    learning_rate=1e-3,                                # 学习率\n",
    "    num_train_epochs=1,                                # 训练轮数\n",
    "    lr_scheduler_type=\"linear\",                        # 学习率调度器类型\n",
    "    warmup_ratio=0.1,                                  # 预热比例\n",
    "    logging_steps=100,                                 # 日志记录步数\n",
    "    save_strategy=\"steps\",                             # 模型保存策略\n",
    "    save_steps=10,                                    # 模型保存步数\n",
    "    # evaluation_strategy=\"steps\",                       # 评估策略\n",
    "    # eval_steps=500,                                    # 评估步数\n",
    "    optim=\"adamw_torch\",                               # 优化器类型\n",
    "    fp16=True,                                        # 是否使用混合精度训练\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ab7d20de-eb50-4494-b7b0-d2639fcd8421",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1790' max='1790' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1790/1790 16:30:37, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>3.195200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>3.194700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>3.205300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>3.179900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>3.154500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>3.122500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>3.124200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>3.096000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>3.106700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>3.089600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>3.070300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>3.059700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>3.051000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>3.042200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>3.028100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>3.026400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>3.018000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/david/anaconda3/envs/peft/lib/python3.10/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1790, training_loss=3.099606988150314, metrics={'train_runtime': 59467.5974, 'train_samples_per_second': 1.927, 'train_steps_per_second': 0.03, 'total_flos': 6.921361834296607e+17, 'train_loss': 3.099606988150314, 'epoch': 0.9995811810693843})"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e03f3c3a-7443-4c5c-a031-b17118118988",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.model.save_pretrained(f\"../models/{model_name}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
